package cn.ac.iscas.webpage.extraction;

import java.io.File;

/**
 * @author Simon@itechs
 *
 */
public class WEStarter {
	public static void main(String[] args){
		//init
		

		
		//traverse dir without sub-dir
		File dir = new File("d:\\test\\");
		
        File file[] = dir.listFiles();
        WebPageExtractor.init(file.length, file);
   //     for (int i = 0; i < WebPageExtractor.threadNumber; i++){
        WebPageExtractor[] we = new WebPageExtractor[WebPageExtractor.threadNumber];
        for (int i = 0; i < WebPageExtractor.threadNumber; i++){
        	System.out.println("Crawler " + i + " Generated");
        	we[i] = new WebPageExtractor(i, file.length);
        }
		
       for (int i = 0; i < WebPageExtractor.threadNumber; i++){
        	System.out.println("Crawler " + i + " Started");
        	we[i].start();
        }
        
//		dir = new File("d:\\taobao2\\list.taobao.com\\");
//		
//        file = dir.listFiles();
//        WebPageExtractor.init(file.length, file);
//   //     for (int i = 0; i < WebPageExtractor.threadNumber; i++){
//        we = new WebPageExtractor[WebPageExtractor.threadNumber];
//        for (int i = 0; i < WebPageExtractor.threadNumber; i++){
//        	System.out.println("Crawler " + i + " Generated");
//        	we[i] = new WebPageExtractor(i, file.length);
//        }
//		
//        for (int i = 0; i < WebPageExtractor.threadNumber; i++){
//        	System.out.println("Crawler " + i + " Started");
//        	we[i].start();
//        }
		//test
//		we.extractContent("http://detail.tmall.com/item.htm?id=7471878619&prt=1331190188047&prc=1");
//		System.out.println(we.toString());
//		we.insertIntoDB();
	}
}